# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html


import datetime
import re
from w3lib.html import remove_tags

import scrapy
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose, TakeFirst, Join

class ArticlespiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


def add_jobbole(value):
    return value+"-bobby"


def date_convert(value):
    try:
        create_date = datetime.datetime.strptime(value, "%Y/%m/%d").date()
    except Exception as e:
        create_date = datetime.datetime.now().date()

    return create_date

def get_nums(value):
    match_re = re.match(".*?(\d+).*", value)
    if match_re:
        nums = int(match_re.group(1))
    else:
        nums = 0

    return nums

def return_value(value):
    return value

def return_amount(value):
    if value:
        return value
    else:
        return ""

#把对应的内容，去掉标签、空格、空行
def go_remove_tag(value):
    content = remove_tags(value)
    return re.sub(r'[\t\r\n\s]', '', content)


def remove_comment_tags(value):
    #去掉tag中提取的评论
    if "评论" in value:
        return ""
    else:
        return value


class ArticleItemLoader(ItemLoader):
    #自定义itemloader
    default_output_processor = TakeFirst()


class JobBoleArticleItem(scrapy.Item):
    title = scrapy.Field()
    create_date = scrapy.Field(
        input_processor=MapCompose(date_convert),
    )
    url = scrapy.Field()
    url_object_id = scrapy.Field()
    front_image_url = scrapy.Field(
        output_processor=MapCompose(return_value)
    )
    front_image_path = scrapy.Field()
    praise_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    comment_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    fav_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    tags = scrapy.Field(
        input_processor=MapCompose(remove_comment_tags),
        output_processor=Join(",")
    )
    content = scrapy.Field()

class SzhrssArticleItem(scrapy.Item):
    title = scrapy.Field()
    create_date = scrapy.Field()


class SzstiArticleItem(scrapy.Item):

    # title = scrapy.Field()
    # create_date = scrapy.Field(
    #      input_processor=MapCompose(date_convert),
    #  )
    url = scrapy.Field()

    # def get_insert_sql(self):
    #     insert_sql = """
    #         insert into szsti_all(title, create_date, url)
    #          VALUES (%s, %s, %s)
    #
    #      """
    #     params = (
    #         self['title'], self['create_date'], self['url']
    #     )
    #     return insert_sql, params


    # 时间格式化
    create_date = scrapy.Field(
        input_processor=MapCompose(date_convert),
    )
    policy_name = scrapy.Field()
    source_department = scrapy.Field(
        input_processor=MapCompose(go_remove_tag)
    )
    project_name = scrapy.Field(
        input_processor=MapCompose(go_remove_tag)
    )
    corporation = scrapy.Field(
        input_processor=MapCompose(go_remove_tag)
    )
    amount =  scrapy.Field(
        input_processor=MapCompose(return_amount)
    )
    leader = scrapy.Field()
    title = scrapy.Field()

    def get_insert_sql(self):
        insert_sql = """
            insert into szsti_list(project_name, corporation, policy_name)
             VALUES (%s, %s, %s)

         """
        params=(
            self['project_name'],self['corporation'], self['policy_name'],
        )
        return insert_sql, params